#!/bin/bash

hyper_id=${1}
CWD=$(pwd)
if [[ ${hyper_id} == "" ]]
then
  echo "usage: ./check_hyper.sh <hyper_id>"
  exit 1
fi
if [ ! -d "${CWD}/logs" ];
then
  mkdir ${CWD}/logs
fi
log_path="${CWD}/logs/check_${hyper_id}-$(date +%Y%m%d-%H%M).log"
detail_log="${CWD}/logs/detail_check_${hyper_id}-$(date +%Y%m%d-%H%M).log"

function fcomp() {
    awk -v n1=$1 -v n2=$2 'BEGIN{ if (n1<n2) exit 0; exit 1}'
}

function get_os_basic_info()
{
    # get distro
    if ssh -Tq ${hyper_id} 'ls /etc/lsb-release' &>/dev/null; then
        os_distro=$(ssh -Tq ${hyper_id} 'lsb_release -d' | awk '{print $2}')
    elif ssh -Tq ${hyper_id} 'ls /etc/kylin-release' &>/dev/null; then
        os_distro=$(ssh -Tq ${hyper_id} 'cat /etc/kylin-release' | awk '{print $1}')
    elif ssh -Tq ${hyper_id} 'ls /etc/redhat-release' &>/dev/null; then
        os_distro=$(ssh -Tq ${hyper_id} 'cat /etc/redhat-release' | awk '{print $1}')
    fi
}

check_hyper_alive(){
  hyper_exist=$(grep ${hyper_id} /etc/hosts|wc -l)
  if [[ ${hyper_exist} == 0 ]]
  then
    echo "ERROR: ${hyper_id}在/etc/hosts中不存在."
    exit 1
  fi
  check_status=$(ssh -Tq ${hyper_id} "supervisorctl status|grep -v RUNNING")
  {
    echo "============================"
    echo "=====checking supervisor===="
    echo "============================"
  } >>${detail_log}
  if [[ $? != 0 ]] || [[ $(echo "${check_status}"|tr -d '\n'|wc -l) != 0 ]]
  then
    echo "ERROR: ${check_status}" >>${detail_log}
    echo "supervisor unhealthy" >>${log_path}
  else
    echo "supervisor healthy" >>${log_path}
  fi
}

function check_nic_status_errors()
{
  {
    echo "============================"
    echo "====checking nic status====="
    echo "============================"
  } >>${detail_log}
    if [[ ${os_distro} == "CentOS" ]] || [[ ${os_distro} == "Kylin" ]]; then
        interfaces=($(ssh -Tq ${hyper_id} "grep -Hw 'ONBOOT=yes' /etc/sysconfig/network-scripts/*" | awk -F ':' '{print $1}' | awk -F '-' '{print $NF}' | grep -v "$(ls /sys/devices/virtual/net)"))
    else
        if [[ ! -f /etc/rc.local.spine_leaf ]]; then
            interfaces=($(ssh -Tq ${hyper_id} "cat /etc/network/interfaces" | grep '^auto' | egrep -v 'lo|br|bond|vlan' | awk '{print $2}'))
        else
            interfaces=($(ssh -Tq ${hyper_id} "cat /etc/network/interfaces.d/*" | grep '^auto' | egrep -v 'lo|br|bond|vlan' | awk '{print $2}'))
        fi
    fi

    d_s_dir="/proc/net/bonding"
    all_slave_devs=();
    if ssh -Tq ${hyper_id} "ls ${d_s_dir}" &>/dev/null; then
        bond_devs=$(ssh -Tq ${hyper_id} "ls -d -1 /proc/net/bonding/*" | cut -d "/" -f 5)
        for bond_dev in ${bond_devs}
        do
            slave_devs=$(ssh -Tq ${hyper_id} "ls -d -1 /sys/class/net/${bond_dev}/{slave,lower}_* 2>/dev/null"|cut -d "_" -f 2)
            all_slave_devs=(${all_slave_devs[@]} ${slave_devs[@]})
        done
    fi

    for interface in "${interfaces[@]}"
    do
        echo "${all_slave_devs[@]}" | grep -wq ${interface}
        if [[ $? -ne 0 ]]; then
            ssh -Tq ${hyper_id} "ip link show dev ${interface}" | grep 'state UP ' > /dev/null
            if [[ $? -ne 0 ]]; then
              echo "ERROR: nic_status: ${interface} is down!" >> ${detail_log}
              echo "${interface}_status unhealthy" >> ${log_path}
            else
              echo "INFO: nic_status: ${interface} is up." >> ${detail_log}
              echo "${interface}_status healthy" >> ${log_path}
            fi
        fi

        rx_crc_errors=$(ssh -Tq ${hyper_id} "cat /sys/class/net/${interface}/statistics/rx_crc_errors")
        rx_errors=$(ssh -Tq ${hyper_id} "cat /sys/class/net/${interface}/statistics/rx_errors")
        rx_frame_errors=$(ssh -Tq ${hyper_id} "cat /sys/class/net/${interface}/statistics/rx_frame_errors")
        tx_errors=$(ssh -Tq ${hyper_id} "cat /sys/class/net/${interface}/statistics/tx_errors")
        sleep 5
        curr_rx_crc_errors=$(ssh -Tq ${hyper_id} "cat /sys/class/net/${interface}/statistics/rx_crc_errors")
        curr_rx_errors=$(ssh -Tq ${hyper_id} "cat /sys/class/net/${interface}/statistics/rx_errors")
        curr_rx_frame_errors=$(ssh -Tq ${hyper_id} "cat /sys/class/net/${interface}/statistics/rx_frame_errors")
        curr_tx_errors=$(ssh -Tq ${hyper_id} "cat /sys/class/net/${interface}/statistics/tx_errors")
        ((rx_crc_difference=${curr_rx_crc_errors}-${rx_crc_errors}))
        ((rx_difference=${curr_rx_errors}-${rx_errors}))
        ((rx_frame_difference=${curr_rx_frame_errors}-${rx_frame_errors}))
        ((tx_difference=${curr_tx_errors}-${tx_errors}))
        ((all_diff=${rx_crc_difference}+${rx_difference}+${rx_frame_difference}+${tx_difference}))
        if [[ ${all_diff} -gt 10 ]]
        then
          {
          echo "ERROR: ${interface}_rx_crc_errors: ${rx_crc_difference}"
          echo "ERROR: ${interface}_rx_errors: ${rx_difference}"
          echo "ERROR: ${interface}_rx_frames: ${rx_frame_difference}"
          echo "ERROR: ${interface}_tx_crc_errors: ${tx_difference}"
          } >> ${detail_log}
          echo "${interface}_error_num unhealthy" >> ${log_path}
        else
          echo "${interface}_error_num healthy" >> ${log_path}
        fi
    done
}

check_disk_errors(){
  disk_info=$(ssh -Tq ${hyper_id} "disktool show -l pd"|grep -E '^\|   [0-9]')
  media_error=$(echo "${disk_info}"|grep -oE '[0-9999]\|[0-9999]\|[0-9999]'|awk -F'|' 'BEGIN{sum=0}{sum=sum+$1}END{print sum}')
  other_error=$(echo "${disk_info}"|grep -oE '[0-9999]\|[0-9999]\|[0-9999]'|awk -F'|' 'BEGIN{sum=0}{sum=sum+$1}END{print sum}')
  predictive_failure_error=$(echo "${disk_info}"|awk -F'|' 'BEGIN{sum=0}{sum=sum+$1}END{print sum}')
  if [[ ${media_error} != 0 ]] || [[ ${other_error} != 0 ]] || [[ ${predictive_failure_error} != 0 ]]; then
    echo "ERROR: ${disk_info}" >>${detail_log}
    echo "disk_error_num unhealthy" >>${log_path}
  else
    echo "disk_error_num healthy" >>${log_path}
  fi
}

check_disk(){
  {
    echo "============================"
    echo "=======checking disk========"
    echo "============================"
  } >>${detail_log}
  if ssh -Tq ${hyper_id} "ls /pitrix/conf/repl_container.flag" &>/dev/null; then
    zpool_status=$(ssh -Tq ${hyper_id} "zpool status")
    repl_status=$(ssh -Tq ${hyper_id} "repl-overview -v | grep -i -C 2 InitSyncing")
    if [[ $(echo "${zpool_status}"|grep 'No known data errors'|wc -l) != 1 ]]; then
      echo "ERROR: ${zpool_status}" >>${detail_log}
      echo "zpool unhealthy" >> ${log_path}
    else
      echo "zpool healthy" >> ${log_path}
    fi
    if [[ $(echo "${repl_status}"|tr -d "\n"|wc -l) != 0 ]]; then
      echo "ERROR: ${repl_status}" >>${detail_log}
      echo "repl-overview unhealthy" >> ${log_path}
    else
      echo "repl-overview healthy" >> ${log_path}
    fi
  fi

  if ssh -Tq ${hyper_id} "which disktool" &>/dev/null; then
    check_disk_errors
  else
    if [[ ${os_distro} == "CentOS" ]]; then
      scp ${CWD}/disktool-2.0.0-202212141611.el7.x86_64.rpm ${hyper_id}: &>/dev/null
      ssh -Tq ${hyper_id} "rpm -ivh /root/disktool-2.0.0-202212141611.el7.x86_64.rpm" &>/dev/null
      if [[ $? != 0 ]];then echo "ERROR:_Install_CentOS_disktool_failed." >>${log_path}; return 0;fi
    elif [[ ${os_distro} == "Kylin" ]]; then
      echo "ERROR:_Kylin_system_is_not_support." >>${log_path}
      return 0
    else
      scp ${CWD}/disktool_2.0.0-202211281016_amd64.deb ${hyper_id}: &>/dev/null
      ssh -Tq ${hyper_id} "dpkg -i /root/disktool_2.0.0-202211281016_amd64.deb" &>/dev/null
      if [[ $? != 0 ]];then echo "ERROR:_Install_Ubuntu_disktool_failed." >>${log_path}; return 0;fi
    fi
    check_disk_errors
  fi
}

check_process(){
  {
    echo "============================"
    echo "======checking process======"
    echo "============================"
  } >>${detail_log}
  R_num=$(ssh -Tq ${hyper_id} "ps -aux|grep -v COMMAND|grep ' R' |grep -v grep|wc -l")
  D_num=$(ssh -Tq ${hyper_id} "ps -aux|grep -v COMMAND|grep ' D '|grep -v grep")
  Z_num=$(ssh -Tq ${hyper_id} "ps -aux|grep -v COMMAND|grep ' Z '|grep -v grep")
  X_num=$(ssh -Tq ${hyper_id} "ps -aux|grep -v COMMAND|grep ' X '|grep -v grep")
  if [[ $(echo "${D_num}"|tr -d '\n'|wc -l) != 0 ]] || [[ $(echo "${D_num}"|tr -d '\n'|wc -l) != 0 ]] || [[ $(echo "${D_num}"|tr -d '\n'|wc -l) != 0 ]]
  then
    {
      echo "ERROR: R process num: ${R_num}."
      echo "ERROR: D process: ${D_num}."
      echo "ERROR: Z process: ${Z_num}."
      echo "ERROR: X process: ${X_num}."
    } >>${detail_log}
    echo "process unhealthy" >>${log_path}
  else
    echo "process healthy" >>${log_path}
  fi
}

check_cpu(){
  {
    echo "============================"
    echo "========checking cpu========"
    echo "============================"
  } >>${detail_log}
  line=$(ssh -Tq ${hyper_id} "top -b -d 0.1 -n 2 | grep '^Cpu\|^%Cpu' | awk -F','  '{print \$4, \$5, \$7}' | awk 'NF > 0' | tail -1")

  idle=$(echo $line | awk '{print $1}' | awk -F'%' '{print $1}')
  idle=${idle/.*}
  ((cpu=100-$idle))
  if [[ $cpu -ge 90 ]]; then
    echo "ERROR: cpu_usage [${cpu}] is above 90." >>${detail_log}
    echo "cpu_usage unhealthy" >>${log_path}
  else
    echo "cpu_usage healthy" >>${log_path}
  fi
  iowait=$(echo $line | awk '{print $2}' | awk -F'%' '{print $1}')
  # on 14.04 $line return sth like "98.6 id   0.0 wa   0.0 si"
  if [ "x$iowait" = "xid" ]; then
      iowait=$(echo $line | awk '{print $3}')
  fi
  iowait=${iowait/.*}
  if [[ $iowait -ge 30 ]]; then
    echo "ERROR: iowait [${iowait}] is above 30." >>${detail_log}
    echo "iowait unhealthy" >>${log_path}
  else
    echo "iowait healthy" >>${log_path}
  fi

  si=$(ssh -Tq ${hyper_id} "top -b -d 0.1 -n 2 -w 256 | grep ksoftirqd | head -1" | awk '{print $9}')
  si=${si/.*}
  if [[ $si -ge 5 ]]; then
    echo "ERROR: si [${si}] is above 10." >>${detail_log}
    echo "si unhealthy" >>${log_path}
    if [ -f /pitrix/bin/net-flow ]; then
        /pitrix/bin/net-flow -c 2 >>${detail_log}
    fi
  else
    echo "si healthy" >>${log_path}
  fi

  fifteenmin_load=$(ssh -Tq ${hyper_id} "uptime" | awk  '{print $NF}')
  cpu_num=$(ssh -Tq ${hyper_id} "grep 'model name'  /proc/cpuinfo |wc -l")
  load_threshold=${cpu_num}
  fcomp ${fifteenmin_load} ${load_threshold}
  if [[ $? -eq 1 ]]; then
    echo "ERROR: load [${fifteenmin_load}] is above [${load_threshold}]." >>${detail_log}
    echo "load_average unhealthy" >>${log_path}
  else
    echo "load_average healthy" >>${log_path}
  fi

  arc_usage=$(ssh -Tq ${hyper_id} 'top -b -n 1|grep arc_reclaim'|awk '{print $9}')
  if [[ ${arc_usage} == "" ]];then arc_usage=0;fi
  if [[ $(echo "${arc_usage} > 90"|bc) -eq 1 ]];then
    echo "ERROR: arc_reclaim [${arc_usage}] is above 90." >>${detail_log}
    echo "arc_reclaim unhealthy" >>${log_path}
  else
    echo "arc_reclaim healthy" >>${log_path}
  fi
}

check_swap_usage(){
  {
    echo "============================"
    echo "=======checking swap========"
    echo "============================"
  } >>${detail_log}
  swap_usage=$(ssh -Tq ${hyper_id} "free -m | grep 'Swap:'"| awk '{print (1-$3/$2)*100}')
  if [[ $(echo "${swap_usage} > 15"|bc) -eq 1 ]]; then
    echo "ERROR: swap_usage [${swap_usage}] is above 15%." >>${detail_log}
    echo "swap_usage unhealthy" >>${log_path}
  else
    echo "swap_usage healthy" >>${log_path}
  fi
}

check_kern_log(){
  {
    echo "============================"
    echo "=======checking kern========"
    echo "============================"
  } >>${detail_log}
  kern_log=$(ssh -Tq ${hyper_id} "tail -n 1000 /var/log/kern.log|grep -iE 'error|call trace|unrecovered|route cache is full|max_size'")
  if [[ $(echo "${kern_log}"|tr -d '\n'|wc -l) != 0 ]];then
    echo "ERROR: ${kern_log}" >>${detail_log}
    echo "kern_log unhealthy" >>${log_path}
  else
    echo "kern_log healthy" >>${log_path}
  fi
}


get_os_basic_info

{
    echo "===================================="
    echo "======check_${hyper_id}_status======"
    echo "===================================="
  } >>${log_path}

check_hyper_alive
check_cpu
check_nic_status_errors
check_disk
check_process
check_swap_usage
check_kern_log

column -t ${log_path}
echo "You can view more detail log in ${detail_log}"

